package com.zzh.jsoupCrawler;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
import org.junit.Test;

/**
 * @author zzh
 * @description 天眼查爬虫--抓取期货公司数据
 * @date 2017年2月9日 上午11:30:14
 */
public class Crawler {
	public final String Path = "D:/code/file/";
	public final String companyListPath = "D:/code/file/companyList/";
	public final String companyPath = "D:/code/file/company/";

	public static void main(String[] args) throws IOException {
		// TODO Auto-generated method stub
		Crawler crawler = new Crawler();
		// crawler.begin();
		crawler.parseCompanyList();
	}

	private void begin() {
		String url_begin = "http://bj.tianyancha.com/search/p1?key=%E6%9C%9F%E8%B4%A7";
		String html_begin = null; // 网页内容
		try {

			html_begin = getHtmlContext(url_begin);
			// 解析该文本获取查询总页数
			Document doc = Jsoup.parse(html_begin);

			// 获取总页数
			String total_page = null;
			Elements els_total_page = doc.select("div.total.ng-binding");
			for (Element el : els_total_page) {
				System.out.println(el.text());
				total_page = el.text();
			}
			System.out.println(total_page);
			// 正则获取

			int total_num = 0;
			Pattern p = Pattern.compile("\\d+");
			Matcher m = p.matcher(total_page);
			if (m.find()) {
				total_num = Integer.parseInt(m.group());
			}
			System.out.println("总页数：" + total_num);
			// 抓取的网页保存到
			for (int i = 1; i <= total_num; i++) {
				if (1 == i) {
					// 实现数据持久化，写入文档
					FileWriter fw = new FileWriter(companyListPath + i + ".html", true);
					fw.write(html_begin);
					fw.flush();
					fw.close();
					// System.exit(0);

				} else {
					// 处理下几页数据
					String usr2 = "http://bj.tianyancha.com/search/p" + i + "?key=%E6%9C%9F%E8%B4%A7";
					String html = getHtmlContext(usr2);
					FileWriter fw = new FileWriter(companyListPath + i + ".html", true);
					fw.write(html_begin);
					fw.flush();
					fw.close();

				}
				Random random = new Random();
				int time = random.nextInt(1500) + 1000;
				Thread.sleep(time);

			}

			// 解析companyListPath路径下的html文件，读取各个公司详细

		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	/* 解析companyListPath路径下的html文件，读取各个公司详细 */
	@Test
	public void parseCompanyList() {
		File files = new File(companyListPath);
		if (files.isDirectory()) {
			// int num = 61, i = 1;
			int num = 0, i = 1;
			for (String name : files.list()) {

				System.out.println(companyListPath + name);
				String path = companyListPath + name;
				// 解析
				File file = new File(path);
				try {
					Document doc = Jsoup.parse(file, "utf-8");
					Elements els = doc.select("div.search_result_single.ng-scope>div.row>div>a");// css过滤器
					String companyName = null, companyDetailUrl = null;
					for (Element el : els) {
						companyName = el.text();
						companyDetailUrl = el.attr("href");

						// 获取公司详情页
						String html = getHtmlContext(companyDetailUrl);
						File newFile = new File(companyPath + num++ + "_" + companyName + ".html");
						FileWriter fw = new FileWriter(newFile, true);
						fw.write(html);
						fw.flush();
						fw.close();
						System.out.println(num);
						Thread.sleep(3000);
					}

				} catch (Exception e) {
					e.printStackTrace();
				}

			}
		}

	}

	/* 解析companyPath路径下的html文件(公司详细 内容)、获取公司名、电话、邮箱、网址、地址、法人代表、注册资本、状态、注册时间 */

	@Test
	public void parseCompanyHtml() {
		File files = new File(companyPath);
		if (files.isDirectory()) {
			int num=0;
			for (String filename : files.list()) {
				try {
					String path = companyPath + filename;
					File file =new File(path);
					Document doc = Jsoup.parse(file, "utf-8");
					//System.out.println(doc.text());
					// System.exit(0);
					// 获取公司名、电话邮箱、网址、地址
					String companyname=null, phone=null, email=null, url=null, location=null;
					Elements els = doc.select("div.company_info_text");// css过滤器
					for (Element el : els) {
						//System.out.println(el.text());
						// 获取公司名
						companyname = el.select("div>p").text();
						System.out.println(companyname);
						for (Element e : el.select("div>span")) {
							System.out.println(e.text());
							String tmp = e.text();
							if (tmp.indexOf("电话:") != -1) {
								phone = tmp.substring(tmp.indexOf(":")+1);
							} else if (tmp.indexOf("邮箱:") != -1) {
								email = tmp.substring(tmp.indexOf(":")+1);
							} else if (tmp.indexOf("网址:") != -1) {
								url = e.select("span>a.c9.ng-binding.ng-scope").text(); 
								System.out.println(url); 
							} else if (tmp.indexOf("地址:") != -1) {
								location = tmp.substring(tmp.indexOf(":")+1);
							} 
						}
						
						//System.out.println(companyname);

						System.out.println("------------");
					}
					  
					//获取法人、注册资本、公司状态、注册时间
					String name=null,zczb=null,status=null,regTime=null;
					for(Element el:doc.select("div.row.b-c-white.company-content")){
						System.out.println(el.text());
						name = el.select("td.td-legalPersonName-value.c9>p>a").text(); 
						zczb = el.select("td.td-regCapital-value").text();
						status = el.select("td.td-regStatus-value").text();
						regTime = el.select("td.td-regTime-value").text(); 
//					System.out.println(regTime); 
					}
					

					String line =  num+++"\t"+companyname+"\t"+phone+"\t"+email+"\t"+url+"\t"+ location+"\t"+  name+"\t"+zczb+"\t"+status+"\t"+regTime+"\t";
					File fileSum = new File(Path+"Sum.txt");
					FileWriter fw = new FileWriter(fileSum,true);
					fw.write(line+"\n");
					fw.flush();
					fw.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}

		}
	}

	// 读取本地文件测试companylist
	@Test
	public void test01() {
		File file = new File("D:/code/file/companyList/1.html");
		try {
			Document doc = Jsoup.parse(file, "utf-8");
			System.out.println(doc.text());
			// System.exit(0);
			Elements els = doc.select("div.search_result_single.ng-scope>div.row>div>a");// css过滤器
			for (Element el : els) {
				System.out.println(el.text());
				System.out.println(el.attr("href"));
				System.out.println("------------");
			}

			// 获取总页数
			String total_page = null;
			Elements els_total_page = doc.select("div.total.ng-binding");
			for (Element el : els_total_page) {
				System.out.println(el.text());
				total_page = el.text();
			}

			// 正则获取

			int total_num = 0;
			Pattern p = Pattern.compile("\\d+");
			Matcher m = p.matcher(total_page);
			if (m.find()) {
				total_num = Integer.parseInt(m.group());
			}
			System.out.println(total_num);

		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	// 读取本地文件测试单个company
	@Test
	public void test06() {
		File file = new File("D:/code/file/company_old/9_冠通期货有限公司.html");	
		try {
			Document doc = Jsoup.parse(file, "utf-8");
			System.out.println(doc.text());
			// System.exit(0);
			// 获取公司名、电话邮箱、网址、地址
			String companyname=null, phone=null, email=null, url=null, location=null;
			Elements els = doc.select("div.company_info_text");// css过滤器
			for (Element el : els) {
				System.out.println(el.text());
				// 获取公司名
				companyname = el.select("div>p").text();
				System.out.println(companyname);
				for (Element e : el.select("div>span")) {
					System.out.println(e.text());
					String tmp = e.text();
					if (tmp.indexOf("电话:") != -1) {
						phone = tmp.substring(tmp.indexOf(":"));
					} else if (tmp.indexOf("邮箱:") != -1) {
						email = tmp.substring(tmp.indexOf(":"));
					} else if (tmp.indexOf("网址:") != -1) {
						url = e.select("span>a.c9.ng-binding.ng-scope").text(); 
						System.out.println(url); 
					} else if (tmp.indexOf("地址:") != -1) {
						location = tmp.substring(tmp.indexOf(":"));
					} 
				}
				
				System.out.println(companyname);

				System.out.println("------------");
			}
			
			
			
			//获取法人、注册资本、公司状态、注册时间
			String name=null,zczb=null,status=null,regTime=null;
			for(Element el:doc.select("div.row.b-c-white.company-content")){
				System.out.println(el.text());
				name = el.select("td.td-legalPersonName-value.c9>p>a").text(); 
				zczb = el.select("td.td-regCapital-value").text();
				status = el.select("td.td-regStatus-value").text();
				regTime = el.select("td.td-regTime-value").text(); 
//				System.out.println(regTime); 
			}
			
		
			String line =  companyname+"\t"+phone+"\t"+email+"\t"+url+"\t"+ location+"\t"+  name+"\t"+zczb+"\t"+status+"\t"+regTime;
			File fileSum = new File(Path+"Sum.txt");
			FileWriter fw = new FileWriter(fileSum);
			fw.write(line);
			fw.flush();
			fw.close();
			 
			 
			
			// 获取总页数
			String total_page = null;
			Elements els_total_page = doc.select("div.total.ng-binding");
			for (Element el : els_total_page) {
				System.out.println(el.text());
				total_page = el.text();
			}

			// 正则获取

			int total_num = 0;
			Pattern p = Pattern.compile("\\d+");
			Matcher m = p.matcher(total_page);
			if (m.find()) {
				total_num = Integer.parseInt(m.group());
			}
			System.out.println(total_num);

		} catch (Exception e) {
			e.printStackTrace();
		}

	}
	
	@Test 
	public void test07(){
		String line=null ;
		System.out.println(line); 
	}
	/* 格式化输出html文本 */
	@Test
	public void test03() {
		File file = new File("D:/code/file/1.html");
		String html = null;
		try {
			Document doc = Jsoup.parse(file, "utf-8");
			html = doc.toString();
			String text = Jsoup.clean(html, "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false));
			String text2 = "";
			for (String str : text.split("\n")) {
				if (str != null) {
					str = replaceTab(str);
					str = str.replace("\\", "");
					if (!str.trim().equals("")) {
						text2 += str + "\n";
						// System.out.println(str);
					}
					
				}
				
			}
			System.out.println(text2);
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	@Test
	public void test04() {
		String str = "dsa\td\ras\nd  as            dasdasdsa";
		System.out.println(replaceTab(str));

	}

	// 去除制表符
	public String replaceTab(String str) {
		String dest = "";
		try {
			if (str != null) { // \\s*|\t|\r|\n \t
				Pattern p = Pattern.compile("\\s*|\t|\r|\n");
				Matcher m = p.matcher(str);
				dest = m.replaceAll("");
			}
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return dest;
	}

	@Test
	public void test05() {
		String url = "http://www.tianyancha.com/company/25339495";
		System.out.println(getHtmlContext(url));

	}

	/* 根据一个链接获取网页内容 */
	private String getHtmlContext(String url) {
		String context = null;
		String exec = "D:/code/phantomjs-2.1.1-windows/bin/phantomjs D:/code/phantomjs-2.1.1-windows/bin/code.js  "
				+ url;
		Runtime rt = Runtime.getRuntime();
		try {
			Process p = rt.exec(exec);
			InputStream is = p.getInputStream();
			Document doc = Jsoup.parse(is, "UTF-8", url);
			// System.out.println(doc);
			context = doc.toString();
		} catch (IOException e) {
			e.printStackTrace();
		}

		return context;
	}

	/* 根据一个链接获取网页内容 */
	public void test() throws IOException {
		String url = "http://bj.tianyancha.com/search/p2?key=%E6%9C%9F%E8%B4%A7";
		url = "http://www.tianyancha.com/company/25339495";
		Document doc = Jsoup.connect(url).get();
		// Document doc = Jsoup.connect(url).header("Accept", "application/json,
		// text/plain, */*")
		// .header("Accept-Encoding", "gzip, deflate,
		// sdch").header("Accept-Language", "zh-CN,zh;q=0.8")
		// .header("Referer",
		// "http://bj.tianyancha.com/search/%E6%9C%9F%E8%B4%A7.json?&base=bj")
		// .header("User-Agent",
		// "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like
		// Gecko) Chrome/50.0.2661.94 Safari/537.36")
		// .timeout(5000).get();
		System.out.println(doc);

	}

}
